package analisador;

import java.io.File;
import java.io.IOException;
import java.sql.*;
import java.util.*;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Version;

public class SignificanciaMsg {

    static Connection con;
    static Map<Integer, String> mensagens = new HashMap<Integer, String>();
    static Map<String, Integer> frequencia = new HashMap<String, Integer>();
    final static String pathIndexDir = "./arquivosGerados/out/";

    public static void main(String[] args) throws Exception {
        conectar();
        consultar();
        criarIndice();
        calcularFrequencia();
        calcularTF_IDF();
        //retirarTokens(0.3);

        desconectar();
    }

    public static void conectar() throws Exception {
        con = DriverManager.getConnection(
                "jdbc:mysql://localhost:3306/hadoop",
                "root",
                "root");
    }

    private static void consultar() throws SQLException {
        Statement stmt = con.createStatement();
        ResultSet rs = stmt.executeQuery("SELECT tokens, id FROM comentario");
        int branco = 0;
        while (rs.next()) {
            int id = rs.getInt("id");
            String tokens = rs.getString("tokens");
            if (tokens == null) {
                branco++;
            } else {
                mensagens.put(id, tokens);
            }
        }
        rs.close();
        System.out.println(branco);
    }

    private static void criarIndice() throws CorruptIndexException, LockObtainFailedException, IOException {
        Analyzer an = new WhitespaceAnalyzer(Version.LUCENE_35);
        Directory dir = FSDirectory.open(new File(pathIndexDir));

        IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_35, an));
        indexWriter.deleteAll();

        for (Integer id : mensagens.keySet()) {
            Document d = new Document();
            d.add(new Field("id", String.valueOf(id), Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
            d.add(new Field("tokens", mensagens.get(id), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES));
            indexWriter.addDocument(d);
        }
        indexWriter.close();
        dir.close();
    }

    public static void desconectar() throws Exception {
        con.close();
    }

    private static void calcularFrequencia() throws IOException {
        Directory dir = FSDirectory.open(new File(pathIndexDir));
        IndexReader reader = IndexReader.open(dir);
        TermEnum termReader = reader.terms();
        while (termReader.next()) {
            Term t = termReader.term();
            frequencia.put(t.text(), reader.docFreq(t));
        }
        reader.close();
        dir.close();
    }

    private static void calcularTF_IDF() throws SQLException {

        Map<String, Double> score;
        Map<String, Integer> countToken;

        for (Integer i : mensagens.keySet()) {
            countToken = new HashMap<String, Integer>();

            String tokens = mensagens.get(i);

            StringTokenizer tokenizer = new StringTokenizer(tokens);
            while (tokenizer.hasMoreElements()) {
                String token = tokenizer.nextToken();
                if (!countToken.containsKey(token)) {
                    countToken.put(token, 0);
                }
                countToken.put(token, countToken.get(token) + 1);
            }
            score = new HashMap<String, Double>();
            for (String token : countToken.keySet()) {
                if (!frequencia.containsKey(token)) {
                    frequencia.put(token, 1);
                }
                double tf = tf(countToken.get(token), countToken.size());
                double idf = idf(frequencia.get(token), frequencia.size());
                score.put(token, tf * idf);
            }
            Map<String, Double> scoreOrdered = ordenar(score);

            StringBuilder tokenOrdered = new StringBuilder();
            for (String token : scoreOrdered.keySet()) {
                tokenOrdered.append(token).append(" ");
            }
            PreparedStatement p = con.prepareStatement("UPDATE comentario SET TOKENSORDERED=? WHERE ID=?");
            p.setString(1, tokenOrdered.toString());
            p.setInt(2, i);
            p.execute();
            p.close();
            if (i % 1000 == 0) {
                System.out.println(i);
            }
        }
    }

    static double idf(int numDoc, int totalDocuments) {
        return Math.log((double) totalDocuments / (double) numDoc);
    }

    static double tf(int num, int numTerm) {
        return (double) num / (double) numTerm;
    }

    private static Map<String, Double> ordenar(Map<String, Double> score) {
        Object[] entries = score.entrySet().toArray();
        Arrays.sort(entries, new Comparator() {

            public int compare(Object lhs, Object rhs) {
                Map.Entry le = (Map.Entry) lhs;
                Map.Entry re = (Map.Entry) rhs;
                return ((Comparable) re.getValue()).compareTo(
                        (Comparable) le.getValue());
            }
        });
        Map<String, Double> h = new LinkedHashMap<String, Double>();
        for (int i = 0; i < entries.length; i++) {
            Map.Entry entry = (Map.Entry) entries[i];
            h.put((String) entry.getKey(), (Double) entry.getValue());
        }
        return h;
    }

    private static void retirarTokens(double d) throws SQLException {

        Statement stmt = con.createStatement();
        ResultSet rs = stmt.executeQuery("SELECT id, tokensordered FROM comentario");

        while (rs.next()) {
            System.out.println(rs.getInt("id"));
            if (rs.getString("tokensordered")==null){
                continue;
            }
            String[] tokens  = rs.getString("tokensordered").split(" ");
            int q = (int) ( d * tokens.length);
            
            StringBuilder newTokens =  new StringBuilder();
            for(int i = 0; i < q; i++){
                newTokens.append(tokens[i]).append(" ");
            }
            
            PreparedStatement p = con.prepareStatement("UPDATE comentario SET TOKENSORDERED=? WHERE ID=?");
            p.setString(1, newTokens.toString());
            p.setInt(2, rs.getInt("id"));
            p.execute();
            p.close();
        }
    }
}
